bitkeeper revision 1.1159.1.105 (412cbda71t8CNbo2C1IX2_T3wi4QIQ)

Move count_info outside pfn_info union, to where it belongs.

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 8739506b73b3e864edf92a01e1b22a75cd51e27e..3031d5de69db9e3cdadb696590281c2d73efd858 100644 (file)
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -480,7 +480,7 @@ void domain_relinquish_memory(struct domain *d)
     {
         page = list_entry(ent, struct pfn_info, list);
 
-        if ( test_and_clear_bit(_PGC_allocated, &page->u.inuse.count_info) )
+        if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
             put_page(page);
     }
 
@@ -489,10 +489,10 @@ void domain_relinquish_memory(struct domain *d)
     {
         page = list_entry(ent, struct pfn_info, list);
 
-        if ( test_and_clear_bit(_PGC_guest_pinned, &page->u.inuse.count_info) )
+        if ( test_and_clear_bit(_PGC_guest_pinned, &page->count_info) )
             put_page_and_type(page);
 
-        if ( test_and_clear_bit(_PGC_allocated, &page->u.inuse.count_info) )
+        if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
             put_page(page);
 
         /*
@@ -668,9 +668,9 @@ int construct_dom0(struct domain *p,
           mfn++ )
     {
         page = &frame_table[mfn];
-        page->u.inuse.domain     = p;
-        page->u.inuse.type_info  = 0;
-        page->u.inuse.count_info = PGC_always_set | PGC_allocated | 1;
+        page->u.inuse.domain    = p;
+        page->u.inuse.type_info = 0;
+        page->count_info        = PGC_always_set | PGC_allocated | 1;
         list_add_tail(&page->list, &p->page_list);
         p->tot_pages++; p->max_pages++;
     }
@@ -715,7 +715,7 @@ int construct_dom0(struct domain *p,
         *l1tab++ = mk_l1_pgentry((mfn << PAGE_SHIFT) | L1_PROT);
         
         page = &frame_table[mfn];
-        set_bit(_PGC_tlb_flush_on_type_change, &page->u.inuse.count_info);
+        set_bit(_PGC_tlb_flush_on_type_change, &page->count_info);
         if ( !get_page_and_type(page, p, PGT_writable_page) )
             BUG();
 
@@ -739,7 +739,7 @@ int construct_dom0(struct domain *p,
             /* Get another ref to L2 page so that it can be pinned. */
             if ( !get_page_and_type(page, p, PGT_l2_page_table) )
                 BUG();
-            set_bit(_PGC_guest_pinned, &page->u.inuse.count_info);
+            set_bit(_PGC_guest_pinned, &page->count_info);
         }
         else
         {

diff --git a/xen/arch/x86/memory.c b/xen/arch/x86/memory.c
index 52c9dcca8d9e81a9d7aaddf9b65727a837378de0..d2f58b4cfa640194d4ab4b7ce0376f55755faa2e 100644 (file)
--- a/xen/arch/x86/memory.c
+++ b/xen/arch/x86/memory.c
@@ -146,6 +146,23 @@ void arch_init_memory(void)
     static void ptwr_disable(void);
     unsigned long mfn;
 
+    /*
+     * We are rather picky about the layout of 'struct pfn_info'. The
+     * count_info and domain fields must be adjacent, as we perform atomic
+     * 64-bit operations on them. Also, just for sanity, we assert the size
+     * of the structure here.
+     */
+    if ( (offsetof(struct pfn_info, u.inuse.domain) != 
+          (offsetof(struct pfn_info, count_info) + sizeof(u32))) ||
+         (sizeof(struct pfn_info) != 24) )
+    {
+        printk("Weird pfn_info layout (%ld,%ld,%d)\n",
+               offsetof(struct pfn_info, count_info),
+               offsetof(struct pfn_info, u.inuse.domain),
+               sizeof(struct pfn_info));
+        for ( ; ; ) ;
+    }
+
     memset(percpu_info, 0, sizeof(percpu_info));
 
     vm_assist_info[VMASST_TYPE_writable_pagetables].enable =
@@ -154,7 +171,7 @@ void arch_init_memory(void)
         ptwr_disable;
 
     for ( mfn = 0; mfn < max_page; mfn++ )
-        frame_table[mfn].u.inuse.count_info |= PGC_always_set;
+        frame_table[mfn].count_info |= PGC_always_set;
 
     /* Initialise to a magic of 0x55555555 so easier to spot bugs later. */
     memset(machine_to_phys_mapping, 0x55, 4<<20);
@@ -182,9 +199,9 @@ void arch_init_memory(void)
           mfn < virt_to_phys(&machine_to_phys_mapping[1<<20])>>PAGE_SHIFT;
           mfn++ )
     {
-        frame_table[mfn].u.inuse.count_info |= PGC_allocated | 1;
-        frame_table[mfn].u.inuse.type_info   = PGT_gdt_page | 1; /* non-RW */
-        frame_table[mfn].u.inuse.domain      = dom_xen;
+        frame_table[mfn].count_info        |= PGC_allocated | 1;
+        frame_table[mfn].u.inuse.type_info  = PGT_gdt_page | 1; /* non-RW */
+        frame_table[mfn].u.inuse.domain     = dom_xen;
     }
 }
 
@@ -417,11 +434,12 @@ get_page_from_l1e(
          * No need for LOCK prefix -- we know that count_info is never zero
          * because it contains PGC_always_set.
          */
+        ASSERT(test_bit(_PGC_always_set, &page->count_info));
         __asm__ __volatile__(
             "cmpxchg8b %2"
-            : "=a" (e), "=d" (count_info),
-              "=m" (*(volatile u64 *)(&page->u.inuse.domain))
-            : "0" (0), "1" (0), "b" (0), "c" (0) );
+            : "=d" (e), "=a" (count_info),
+              "=m" (*(volatile u64 *)(&page->count_info))
+            : "0" (0), "1" (0), "c" (0), "b" (0) );
         if ( unlikely((count_info & PGC_count_mask) == 0) ||
              unlikely(e == NULL) || unlikely(!get_domain(e)) )
              return 0;
@@ -434,7 +452,7 @@ get_page_from_l1e(
     {
         if ( unlikely(!get_page_type(page, PGT_writable_page)) )
             return 0;
-        set_bit(_PGC_tlb_flush_on_type_change, &page->u.inuse.count_info);
+        set_bit(_PGC_tlb_flush_on_type_change, &page->count_info);
     }
 
     return 1;
@@ -741,7 +759,7 @@ static int mod_l1_entry(l1_pgentry_t *pl1e, l1_pgentry_t nl1e)
 int alloc_page_type(struct pfn_info *page, unsigned int type)
 {
     if ( unlikely(test_and_clear_bit(_PGC_tlb_flush_on_type_change, 
-                                     &page->u.inuse.count_info)) )
+                                     &page->count_info)) )
     {
         struct domain *p = page->u.inuse.domain;
         if ( unlikely(NEED_FLUSH(tlbflush_time[p->processor],
@@ -822,8 +840,8 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
             break;
         }
 
-        if ( unlikely(test_and_set_bit(_PGC_guest_pinned, 
-                                       &page->u.inuse.count_info)) )
+        if ( unlikely(test_and_set_bit(_PGC_guest_pinned,
+                                       &page->count_info)) )
         {
             MEM_LOG("Pfn %08lx already pinned", pfn);
             put_page_and_type(page);
@@ -840,7 +858,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
                     ptr, page->u.inuse.domain);
         }
         else if ( likely(test_and_clear_bit(_PGC_guest_pinned, 
-                                            &page->u.inuse.count_info)) )
+                                            &page->count_info)) )
         {
             put_page_and_type(page);
             put_page(page);
@@ -1007,7 +1025,7 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
          * disappears then the deallocation routine will safely spin.
          */
         nd = page->u.inuse.domain;
-        y  = page->u.inuse.count_info;
+        y  = page->count_info;
         do {
             x = y;
             if ( unlikely((x & (PGC_count_mask|PGC_allocated)) != 
@@ -1022,9 +1040,9 @@ static int do_extended_command(unsigned long ptr, unsigned long val)
             }
             __asm__ __volatile__(
                 LOCK_PREFIX "cmpxchg8b %3"
-                : "=a" (nd), "=d" (y), "=b" (e),
-                "=m" (*(volatile u64 *)(&page->u.inuse.domain))
-                : "0" (d), "1" (x), "b" (e), "c" (x) );
+                : "=d" (nd), "=a" (y), "=c" (e),
+                "=m" (*(volatile u64 *)(&page->count_info))
+                : "0" (d), "1" (x), "c" (e), "b" (x) );
         } 
         while ( unlikely(nd != d) || unlikely(y != x) );
         
@@ -1395,7 +1413,7 @@ void ptwr_reconnect_disconnected(unsigned long addr)
                  l1_pgentry_val(linear_pg_table[(unsigned long)pl2e >>
                                                 PAGE_SHIFT]) >> PAGE_SHIFT,
                  frame_table[pfn].u.inuse.type_info,
-                 frame_table[pfn].u.inuse.count_info,
+                 frame_table[pfn].count_info,
                  frame_table[pfn].u.inuse.domain->domain));
 
     nl2e = mk_l2_pgentry(l2_pgentry_val(*pl2e) | _PAGE_PRESENT);
@@ -1417,7 +1435,7 @@ void ptwr_reconnect_disconnected(unsigned long addr)
     PTWR_PRINTK(("[A] now pl2e %p l2e %08lx              taf %08x/%08x/%u\n",
                  pl2e, l2_pgentry_val(*pl2e),
                  frame_table[pfn].u.inuse.type_info,
-                 frame_table[pfn].u.inuse.count_info,
+                 frame_table[pfn].count_info,
                  frame_table[pfn].u.inuse.domain->domain));
     ptwr_info[cpu].disconnected = ENTRIES_PER_L2_PAGETABLE;
     /* make pt page write protected */
@@ -1535,7 +1553,7 @@ int ptwr_do_page_fault(unsigned long addr)
                 l1_pgentry_t *pl1e;
                 PTWR_PRINTK(("[I] freeing l1 page %p taf %08x/%08x\n", page,
                              page->u.inuse.type_info,
-                             page->u.inuse.count_info));
+                             page->count_info));
                 if (ptwr_info[cpu].writable_idx == PTWR_NR_WRITABLES)
                     ptwr_flush_inactive();
                 ptwr_info[cpu].writables[ptwr_info[cpu].writable_idx] = addr;
@@ -1560,7 +1578,7 @@ int ptwr_do_page_fault(unsigned long addr)
                                                             >> PAGE_SHIFT]) >>
                              PAGE_SHIFT,
                              frame_table[pfn].u.inuse.type_info,
-                             frame_table[pfn].u.inuse.count_info,
+                             frame_table[pfn].count_info,
                              frame_table[pfn].u.inuse.domain->domain));
                 /* disconnect l1 page */
                 nl2e = mk_l2_pgentry((l2_pgentry_val(*pl2e) & ~_PAGE_PRESENT));
@@ -1571,7 +1589,7 @@ int ptwr_do_page_fault(unsigned long addr)
                 PTWR_PRINTK(("[A] now pl2e %p l2e %08lx              "
                              "taf %08x/%08x/%u\n", pl2e, l2_pgentry_val(*pl2e),
                              frame_table[pfn].u.inuse.type_info,
-                             frame_table[pfn].u.inuse.count_info,
+                             frame_table[pfn].count_info,
                              frame_table[pfn].u.inuse.domain->domain));
                 ptwr_info[cpu].writable_l1 = addr;
                 pl1e = map_domain_mem(l2_pgentry_to_pagenr(nl2e) <<

diff --git a/xen/common/dom0_ops.c b/xen/common/dom0_ops.c
index 1f1e911b73461eedb09328e1b57940b62499187c..5ae8924143fadd99b5ff6cecf1f3b16483df54d0 100644 (file)
--- a/xen/common/dom0_ops.c
+++ b/xen/common/dom0_ops.c
@@ -628,7 +628,7 @@ long do_dom0_op(dom0_op_t *u_dom0_op)
                         break;
                     }
 
-                   if ( page->u.inuse.count_info & PGC_guest_pinned )
+                   if ( page->count_info & PGC_guest_pinned )
                        type |= LPINTAB;
                    l_arr[j] |= type;
                     put_page(page);

diff --git a/xen/common/dom_mem_ops.c b/xen/common/dom_mem_ops.c
index 4e9e7425af95b9141e3fdac6691b5efde983c7f9..ba570d91f774ff9bf41b786e9194750065da0f57 100644 (file)
--- a/xen/common/dom_mem_ops.c
+++ b/xen/common/dom_mem_ops.c
@@ -82,12 +82,10 @@ static long free_dom_mem(struct domain *d,
                 return i;
             }
 
-            if ( test_and_clear_bit(_PGC_guest_pinned, 
-                                    &page->u.inuse.count_info) )
+            if ( test_and_clear_bit(_PGC_guest_pinned, &page->count_info) )
                 put_page_and_type(page);
             
-            if ( test_and_clear_bit(_PGC_allocated,
-                                    &page->u.inuse.count_info) )
+            if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
                 put_page(page);
 
             put_page(page);

diff --git a/xen/common/keyhandler.c b/xen/common/keyhandler.c
index 6b5872cfbebbc9f4dab74b01af28bce5d3e881d3..16f811d07fae45bdfdb9dfdbe184c77a4a7f7041 100644 (file)
--- a/xen/common/keyhandler.c
+++ b/xen/common/keyhandler.c
@@ -82,14 +82,14 @@ void do_task_queues(unsigned char key, void *dev_id,
             {
                 page = list_entry(ent, struct pfn_info, list);
                 printk("Page %08x: caf=%08x, taf=%08x\n",
-                       page_to_phys(page), page->u.inuse.count_info,
+                       page_to_phys(page), page->count_info,
                        page->u.inuse.type_info);
             }
         }
 
         page = virt_to_page(d->shared_info);
         printk("Shared_info@%08x: caf=%08x, taf=%08x\n",
-               page_to_phys(page), page->u.inuse.count_info,
+               page_to_phys(page), page->count_info,
                page->u.inuse.type_info);
                
         printk("Guest: upcall_pend = %02x, upcall_mask = %02x\n", 

diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index 79b8df7452f5cba08e06d3f387c3d8878a1d9e38..faa3b6ec3a2095aafcadda28a873535cb1aa550c 100644 (file)
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -310,9 +310,9 @@ unsigned long alloc_xenheap_pages(int order)
 
     for ( i = 0; i < (1 << order); i++ )
     {
-        pg[i].u.inuse.count_info = PGC_always_set;
-        pg[i].u.inuse.domain     = NULL;
-        pg[i].u.inuse.type_info  = 0;
+        pg[i].count_info        = PGC_always_set;
+        pg[i].u.inuse.domain    = NULL;
+        pg[i].u.inuse.type_info = 0;
     }
 
     return (unsigned long)page_to_virt(pg);
@@ -383,9 +383,9 @@ struct pfn_info *alloc_domheap_pages(struct domain *d, int order)
             }
         }
 
-        pg[i].u.inuse.count_info = PGC_always_set;
-        pg[i].u.inuse.domain     = NULL;
-        pg[i].u.inuse.type_info  = 0;
+        pg[i].count_info        = PGC_always_set;
+        pg[i].u.inuse.domain    = NULL;
+        pg[i].u.inuse.type_info = 0;
     }
 
     if ( d == NULL )
@@ -411,7 +411,7 @@ struct pfn_info *alloc_domheap_pages(struct domain *d, int order)
     {
         pg[i].u.inuse.domain = d;
         wmb(); /* Domain pointer must be visible before updating refcnt. */
-        pg[i].u.inuse.count_info |= PGC_allocated | 1;
+        pg[i].count_info |= PGC_allocated | 1;
         list_add_tail(&pg[i].list, &d->page_list);
     }
 

diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h
index c07235fa5e70c5cf1102315f5cf7243ce45da210..7bbe5fe06dce38c43c4c4a0327179db018f77528 100644 (file)
--- a/xen/include/asm-x86/mm.h
+++ b/xen/include/asm-x86/mm.h
@@ -30,25 +30,24 @@ struct pfn_info
     /* Each frame can be threaded onto a doubly-linked list. */
     struct list_head list;
 
+    /* Reference count and various PGC_xxx flags and fields. */
+    u32 count_info;
+
     /* Context-dependent fields follow... */
     union {
 
-        /* Page is in use by a domain. */
+        /* Page is in use: ((count_info & PGC_count_mask) != 0). */
         struct {
-            /* Owner of this page. */
+            /* Owner of this page (NULL if page is anonymous). */
             struct domain *domain;
-            /* Reference count and various PGC_xxx flags and fields. */
-            u32 count_info;
             /* Type reference count and various PGT_xxx flags and fields. */
             u32 type_info;
         } inuse;
 
-        /* Page is on a free list. */
+        /* Page is on a free list: ((count_info & PGC_count_mask) == 0). */
         struct {
             /* Mask of possibly-tainted TLBs. */
             unsigned long cpu_mask;
-            /* Must be at same offset as 'u.inuse.count_flags'. */
-            u32 __unavailable;
             /* Order-size of the free chunk this page is the head of. */
             u8 order;
         } free;
@@ -108,8 +107,8 @@ struct pfn_info
         wmb(); /* install valid domain ptr before updating refcnt. */       \
         spin_lock(&(_dom)->page_alloc_lock);                                \
         /* _dom holds an allocation reference */                            \
-        ASSERT((_pfn)->u.inuse.count_info == PGC_always_set);               \
-        (_pfn)->u.inuse.count_info |= PGC_allocated | 1;                    \
+        ASSERT((_pfn)->count_info == PGC_always_set);                       \
+        (_pfn)->count_info |= PGC_allocated | 1;                            \
         if ( unlikely((_dom)->xenheap_pages++ == 0) )                       \
             get_knownalive_domain(_dom);                                    \
         list_add_tail(&(_pfn)->list, &(_dom)->xenpage_list);                \
@@ -126,13 +125,13 @@ void free_page_type(struct pfn_info *page, unsigned int type);
 
 static inline void put_page(struct pfn_info *page)
 {
-    u32 nx, x, y = page->u.inuse.count_info;
+    u32 nx, x, y = page->count_info;
 
     do {
         x  = y;
         nx = x - 1;
     }
-    while ( unlikely((y = cmpxchg(&page->u.inuse.count_info, x, nx)) != x) );
+    while ( unlikely((y = cmpxchg(&page->count_info, x, nx)) != x) );
 
     if ( unlikely((nx & PGC_count_mask) == 0) )
         free_domheap_page(page);
@@ -142,7 +141,7 @@ static inline void put_page(struct pfn_info *page)
 static inline int get_page(struct pfn_info *page,
                            struct domain *domain)
 {
-    u32 x, nx, y = page->u.inuse.count_info;
+    u32 x, nx, y = page->count_info;
     struct domain *p, *np = page->u.inuse.domain;
 
     do {
@@ -160,9 +159,9 @@ static inline int get_page(struct pfn_info *page,
         }
         __asm__ __volatile__(
             LOCK_PREFIX "cmpxchg8b %3"
-            : "=a" (np), "=d" (y), "=b" (p),
-              "=m" (*(volatile u64 *)(&page->u.inuse.domain))
-            : "0" (p), "1" (x), "b" (p), "c" (nx) );
+            : "=d" (np), "=a" (y), "=c" (p),
+              "=m" (*(volatile u64 *)(&page->count_info))
+            : "0" (p), "1" (x), "c" (p), "b" (nx) );
     }
     while ( unlikely(np != p) || unlikely(y != x) );
 
@@ -254,7 +253,7 @@ static inline int get_page_type(struct pfn_info *page, u32 type)
             DPRINTK("Error while validating pfn %08lx for type %08x."
                     " caf=%08x taf=%08x\n",
                     page_to_pfn(page), type,
-                   page->u.inuse.count_info,
+                   page->count_info,
                    page->u.inuse.type_info);
             put_page_type(page);
             return 0;
@@ -292,7 +291,7 @@ static inline int get_page_and_type(struct pfn_info *page,
     ASSERT(((_p)->u.inuse.type_info & PGT_type_mask) == (_t)); \
     ASSERT(((_p)->u.inuse.type_info & PGT_count_mask) != 0)
 #define ASSERT_PAGE_IS_DOMAIN(_p, _d)                          \
-    ASSERT(((_p)->u.inuse.count_info & PGC_count_mask) != 0);  \
+    ASSERT(((_p)->count_info & PGC_count_mask) != 0);          \
     ASSERT((_p)->u.inuse.domain == (_d))
 
 int check_descriptor(unsigned long *d);

diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h
index cbfca412b7bedc7f77de3d3e1470f2b7a4f4f9d2..f874b6be4a9df5f97c04583742aa60e6124509b6 100644 (file)
--- a/xen/include/asm-x86/shadow.h
+++ b/xen/include/asm-x86/shadow.h
@@ -121,7 +121,7 @@ static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn )
                mfn, pfn, m->shadow_dirty_bitmap_size, m );
         SH_LOG("dom=%u caf=%08x taf=%08x\n", 
                frame_table[mfn].u.inuse.domain->domain,
-               frame_table[mfn].u.inuse.count_info, 
+               frame_table[mfn].count_info, 
                frame_table[mfn].u.inuse.type_info );
     }